# --------------------------------------------------------------
## 
# @brief hot question finder
# ---------------------------------------------------------------
#!/usr/bin/python
#coding=gbk

import sys
import time
import MySQLdb
from operator import itemgetter
import shutil

class HotFinder:
    """ hot q finder """

    def __init__(self, textdir):
        self.host='172.23.150.99'
        self.user='root'
        self.passwd='zhishisousuo'
        self.db='hotq'
        self.charset='utf8'    
        
        self.minHotfreq = 2
        
        self.textdir = textdir
        self.indate = 0
        self.inhour = 0
        self.getDate()
                
        self.yest_ask_dic = {}
        self.yest_gold_title_dic = {}
                    
        self.ask_dic = {}
        self.gold_title_dic = {}

        self.conn = 0
        self.cursor = 0
        self.TOP_N = 100

        self.initdb()
    
    def initdb(self):
        try:
            self.conn = MySQLdb.connect(host=self.host,user=self.user,passwd=self.passwd,db=self.db, charset=self.charset)
        except Exception, e:
            print e
            sys.exit()
        self.cursor = self.conn.cursor()
        
        sql = """create table if not exists 
         hotq_%s (
          rankid int(11) not null,
          gid int(11) not null,
          title varchar(1024) default NULL,
          cid int(11) not null default '0',
          freq int(11) not null default '0',
          pvrank int(11) not null default '0',          
          oldfreq int(11) not null default '0',          
          oldpvrank int(11) not null default '0',          
          hotvalue int(11) not null default '0',
          indate int(11) not null,
          inhour int(11) not null,
          ansflag int(11) not null default '0',
          hotflag int(11) not null default '0',
          unused1 int(11) not null default '0',
          unused2 int(11) not null default '0',          
          PRIMARY KEY (rankid, indate, inhour)
          ) ENGINE=MyISAM  default charset=utf8;""" % str(self.indate)[:6]

        try:
           self.cursor.execute(sql)
        except Exception, e:
            print e
            
    def getDate(self):
        """ 2009081215 """
        myDate=time.strftime("%Y%m%d%H", time.localtime())
        print myDate
        self.indate = int(myDate[:8])
        self.inhour = int(myDate[8:])    

    def getAskQuestion(self, askfile, ask_dic):
        try:
            infile = open(askfile, 'r')
        except Exception, e:
            print e
            return 0
        i = 0
        ten = []
        for x in infile:
 #           i += 1
 #           if i > 200:
 #               break
            tmp = x.strip().split('\t')
            if len(tmp) ==3:
                tmp.insert(2, '0')
            if len(tmp) != 4 or tmp[0] == '0':
                continue
            x = '\t'.join(tmp)
            try:
                ten.index(tmp[1])
                continue
            except:
                pass    
            if len(ten) >= 100:
                ten.pop(0)
            ten.append(tmp[1])
            ask_dic.setdefault(tmp[1], []).append(x.strip())
        infile.close()

    def getGoldTitle(self, ask_dic, gold_dic, yestflag):
        for x in ask_dic:
            if len(ask_dic[x]) > self.minHotfreq:
                gold_dic[x] = self.getGoldOne(ask_dic[x], yestflag)
        yest_total = sorted([(x, gold_dic[x][4]) for x in gold_dic], key= itemgetter(1), reverse=True)
        i = 0
        for x in yest_total:
            i += 1
            
            #print gold_dic[x[0]]
            gold_dic[x[0]].append(i) # origin rank

    def getGoldOne(self, titles, yestflag):
        ansflag = 0
        gid = 0
        freq = 0
        cid = 0
        if yestflag:
            freq = int((24.0/self.inhour)*len(titles))
        else:
            freq = len(titles)
        tmp = titles[0].split('\t')
        goldtitle = ''
        if len(tmp) == 4:
            gid = int(tmp[1])
            if gid == 0:
                cid = 0
            else:
                cid = int(tmp[2])
            ansflag = int(tmp[0])                
            goldtitle = tmp[3]
        tmpdic = {}
        titles = [x.split('\t')[3] for x in titles]
        for x in titles:
            if len(x)<10 or len(x)>40:
                continue
            tmpdic[x] = tmpdic.get(x, 0) + 1
        titles = [(x, tmpdic[x]+100-len(x)/2) for x in tmpdic]
        titles = sorted(titles, key=itemgetter(1))        
        if len(titles) > 0:
            goldtitle = titles[-1][0]    
        return [gid, cid, goldtitle, ansflag, freq]
                    
    def dispDic(self, outfile, adic):
        outfile = open(outfile, 'w')
        for x in adic:
            outfile.write(x + '\t' + '\t'.join([str(y) for y in adic[x]]) + '\n')
        outfile.close()

    def statYesterday(self):        
        yest_askfile1 = self.textdir + str(self.indate-1) + '.ask.1'
        yest_askfile2 = self.textdir + str(self.indate-1) + '.ask.2'
        self.getAskQuestion(yest_askfile1, self.yest_ask_dic)
        self.getAskQuestion(yest_askfile2, self.yest_ask_dic)
        self.getGoldTitle(self.yest_ask_dic, self.yest_gold_title_dic, 1)

    def statToday(self):
        askfile1 = self.textdir + str(self.indate) + '.ask.1'
        askfile2 = self.textdir + str(self.indate) + '.ask.2'
        self.getAskQuestion(askfile1, self.ask_dic)
        self.getAskQuestion(askfile2, self.ask_dic)
        self.getGoldTitle(self.ask_dic, self.gold_title_dic, 0)
    
    def getMaxDelta(self, indexid):
        maxdelta = 0
        for x in self.gold_title_dic:
            try:
                tmp = self.gold_title_dic[x][indexid] - self.yest_gold_title_dic[x][indexid] 
            except:
                tmp = self.gold_title_dic[x][indexid]
            if tmp > maxdelta:
                maxdelta = tmp                
        return maxdelta
    
    def getHotValue(self):
        maxRank = max(len(self.gold_title_dic), len(self.yest_gold_title_dic))
        maxRankDelta = self.getMaxDelta(5) 
        maxFreq = max([self.gold_title_dic[x][4] for x in self.gold_title_dic])
        maxFreqDelta = self.getMaxDelta(4) 
        
        freq_weight = float(maxRank+maxRankDelta)/(maxRank+maxRankDelta+maxFreq+maxFreqDelta)
        rank_weight = 1 - freq_weight
        for x in self.gold_title_dic:
            hotv = 0
            try:
                oldrank = maxRank - self.yest_gold_title_dic[x][5]
                originoldrank = self.yest_gold_title_dic[x][5]
            except:
                oldrank = 1
                originoldrank = maxRank 
            newrank = maxRank - self.gold_title_dic[x][5]
            try:
                oldfreq = self.yest_gold_title_dic[x][4]
            except:
                oldfreq = 1
            newfreq = self.gold_title_dic[x][4]
            
            rank_ratio = float(newrank)/maxRank
            rankdelta_ratio = float(newrank - oldrank)/maxRankDelta
            freq_ratio = float(newfreq)/maxFreq
            freqdelta_ratio = float(newfreq - oldfreq)/maxFreqDelta
            
            hotv = int((rank_weight*rank_ratio*rankdelta_ratio + freq_weight*freq_ratio*freqdelta_ratio)*10000)
            self.gold_title_dic[x].extend([oldfreq, originoldrank, hotv])
            
        total = sorted([(self.gold_title_dic[x][8],self.gold_title_dic[x]) for x in self.gold_title_dic], reverse=True)
        total = [x[1] for x in total]
        return total
    
    def loadYestDic(self, yestfile):
        infile = open(yestfile, 'r')
        for x in infile:
            tmp = x.strip().split('\t')
            if len(tmp) > 1:
                tmp[1] = int(tmp[1])
                tmp[3] = int(tmp[2])
                tmp[4] = int(tmp[4])
                tmp[5] = int(tmp[5])
                tmp[6] = int(tmp[6])
                self.yest_gold_title_dic.setdefault(tmp[0], tmp[1:])
        infile.close()
        
    def doFinder(self, starthour):
        """ 
         0     1     2          3       4     5    (output yestdic)||   6         7      8
        [gid, cid, goldtitle, ansflag, freq, rank, (output yestdic)|| oldfreq, oldrank, hotv] """
        yest_total = []
        total = []
        if self.inhour > starthour:
            print "BEGIN START STAT!"
            self.statYesterday()
            self.dispDic('yesdic.txt', self.yest_gold_title_dic)
        else:
            print "BEGIN LOAD OLD!"
            self.loadYestDic('yesdic.txt')
        self.statToday()
        total = self.getHotValue()
        
        rankid = 1       
        for x in total:
            gid = x[0]
            title = MySQLdb.escape_string(x[2].decode('gbk', 'ignore').encode('utf8'))
            cid = x[1] 
            freq = x[4]
            pvrank = x[5]
            oldfreq = x[6]
            oldpvrank = x[7]
            hotvalue = x[8]
            indate = self.indate
            inhour = self.inhour
            ansflag = x[3]
            hotflag = 0

            sql_query = "replace into hotq_%s(rankid, gid, title, cid, freq, pvrank, oldfreq, oldpvrank, hotvalue, indate, inhour, ansflag, hotflag) values(%d, %d, '%s', %d, %d, %d, %d, %d, %d, %d, %02d, %d, %d)" % \
                      (str(self.indate)[:6], rankid, gid, title, cid, freq, pvrank, oldfreq, oldpvrank, hotvalue, indate, inhour, ansflag, hotflag)
            print sql_query
            try:
                self.cursor.execute(sql_query)
                rankid += 1
                #print rankid
                if rankid > self.TOP_N:
                    break
            except Exception, e:
                print e

if __name__ == '__main__':
    sys.path.append('../lib/')
    textdir = '~/rawdata/'
    hot = HotFinder(textdir)
    if len(sys.argv) == 2:
        hot.doFinder(int(sys.argv[1]))
    elif len(sys.argv) == 1:
        hot.doFinder(0)
    else:
        print "Param error."
            
    print 'ok'
